package com.cike.sparkstudy.sql.scala

import org.apache.spark.sql.{SQLContext, SaveMode}
import org.apache.spark.{SparkConf, SparkContext}

/**
  * 合并元数据，把学生基本信息和学生成绩信息合并
  *
  */
object ParquetMergeSchema {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
      .setMaster("local")
      .setAppName("ParquetMergeSchema")
    val sc = new SparkContext(conf)
    val sqlContext = new SQLContext(sc)


    //需要先导入一个隐式转换
    import sqlContext.implicits._

    //创建第一个DataFrame，学生基本信息
    val studentswithNameAge = Array(("leo",23),("marry",18))
    val studentswithNameAgeDF = sc.parallelize(studentswithNameAge,2).toDF("name","age")
    studentswithNameAgeDF.save("/developerCodes/test/students","parquet",SaveMode.Append)


    //创建第二个DataFrame，学生成绩信息
    val studentsWithNameGrade = Array(("jack","A"),("tom","B"))
    val studentsWithNameGradeDF = sc.parallelize(studentsWithNameGrade,2).toDF("name","grade")
    studentsWithNameGradeDF.save("/developerCodes/test/students","parquet",SaveMode.Append)

    //在这里两个DataFrame中的元数据肯定是不一致的(name,age),(name,grade)
    //我们希望能把两个数据合并成(name,age,grade)的形式显示
    //使用mergeScheme方式合并，读取students中的数据
    val students = sqlContext.read.option("mergeSchema","true")
      .parquet("/developerCodes/test/students")
    students.printSchema()
    students.show()



  }



}
